# -*- coding: utf-8 -*-
'''
Created on Nov 1, 2013

@author: LONG HOANG GIANG
'''
from urlparse import urljoin
import logging
import os
import re
import sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
import lib
import json
import workerpool

logging.basicConfig(level=logging.DEBUG, format='%(levelname)s :: %(asctime)s :: %(message)s', datefmt='%d/%m/%Y %H:%M:%S')

class MyJob(workerpool.Job):
    
    def __init__(self, func, chapter):
        self.func = func
        self.chapter = chapter
        
    def run(self):
        self.func(self.chapter)
        
class VinaWap():
    
    filepath = "/longhoanggiang/truyentext/vinawap"
    
    def __init__(self):
        lib.makedir(self.filepath + "/files")
        
    def filter(self, url):
        ft = ['sieu-pham-co-ban-gia-su-tuoi-day-thi-chap-3-2']
        for ifilter in ft:
            if ifilter in url:
                return True
        return False 
        
    
    def getStories(self, spage, epage):
        data = []
        for page in range(spage, epage+1):
            url = 'http://vinawap.mobi/truyen-sex/' if page == 1 else 'http://vinawap.mobi/truyen-sex/{0}/'.format(page)
            tree = lib.Web.load(url).build_tree()
            for item in tree.xpath("//div[@class='tin-anh']//li/a"):
                name = lib.stringify(item)
                link = item.get('href')
                if link == '': continue
                link = urljoin(url, link)
                if 'hhttp://' in link: link = re.sub(r"hhttp://", 'http://', link)
                print name, link
                data.append({'name': name, 'url': link, 'path': lib.crc32unsigned(link)})
        return data
    
    def getNextPageLink(self, aNode):
        url = ''
        firstPage = True
        for node in aNode:
            if node.tail.strip() == '': continue
            firstPage = False
            nextNode = node.xpath("./following-sibling::a")
            if len(nextNode) == 0: break
        if firstPage and len(aNode) > 0: nextNode = aNode
        
        if len(nextNode) > 0:
            link = nextNode[0].get('href')
            if link != '':
                url = urljoin('http://vinawap.mobi/story/', link)
        return url
    
    def getDetail(self, item):
        url = item['url']
        chapCount = 0
        data = []
        while url != '':
            path = lib.crc32unsigned(url)
            tree = lib.Web.load(url).build_tree()
            contentNode = tree.xpath("//div[@class='anx_article']")[0]
            contentText = lib.html2text(contentNode)
            nextPageNodes = tree.xpath("//div[@class='page']/a")
            nc = len(nextPageNodes)
            if nc > 0:
                chapCount += 1
                url = self.getNextPageLink(nextPageNodes)
                if url != '': 
                    print '================================='
                    print 'Go Next: ', url
                    print '================================='
            else:
                url = ''
            name = item['name'] + " ({0})".format(chapCount) if chapCount > 0 else item['name']
            contentText = "<h4>{0}</h4><div id=\"content\">{1}</div>".format(name, contentText)
            data.append({'name': name, 'path': path})
            print contentText
            
            lib.gz_file_put_content(json.dumps(contentText), self.filepath+"/files/"+path)
            
        return data
    
    def getChapters(self, item):
        xpathStr = r'/html/body/div[7]//li//a' if '/story/' in item['url'] else r'/html/body/div[4]/div[2]/div//li//a'
        tree = lib.Web.load(item['url']).build_tree()
        data = []
        for node in tree.xpath(xpathStr):
            name = lib.stringify(node)
            link = node.get('href')
            print link, name
            if link == '' or self.filter(link): continue
            link = urljoin(item['url'], link)
            data.append({'name': name, 'url': link, 'path': lib.crc32unsigned(link)})
        data.reverse()
        chapters = []
        for chap in data:
            ichap = self.getDetail(chap)
            for iichap in ichap:
                chapters.append(iichap)
        return chapters
    
    def process(self, spage, epage):
        chapters = []
        stories = self.getStories(spage, epage)
        for story in stories:
            cc = self.getChapters(story)
            for iichap in cc:
                chapters.append(iichap)
        lib.gz_file_put_content(lib.encryptCipher(json.dumps(chapters)), self.filepath + "/data")
        
        
if __name__ == '__main__':
    
    url = 'http://vinawap.mobi/truyen-sex/'
    
    t = VinaWap()
#     t.getChapters({'name': '', 'url': 'http://vinawap.mobi/story/sieu-pham-one-way-ticket-2/', 'path': ''})
    t.process(1, 1)
    
    logging.info("finished")
    os._exit(1)        